package cn.book;

import cn.book.bus.utils.JsoupUtil;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.junit.jupiter.api.Test;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.Site;
import us.codecraft.webmagic.Spider;
import us.codecraft.webmagic.processor.PageProcessor;

import java.util.Optional;

public class GithubRepoPageProcessor implements PageProcessor {

    private Site site = Site.me().setRetryTimes(3).setSleepTime(100);

    @Override
    public void process(Page page) {
        page.addTargetRequests(page.getHtml().links().regex("(https://github\\.com/\\w+/\\w+)").all());
        page.putField("author", page.getUrl().regex("https://github\\.com/(\\w+)/.*").toString());
        page.putField("name", page.getHtml().xpath("//h1[@class='entry-title public']/strong/a/text()").toString());
        if (page.getResultItems().get("name")==null){
            //skip this page
            page.setSkip(true);
        }
        page.putField("readme", page.getHtml().xpath("//div[@id='readme']/tidyText()"));
    }

    @Override
    public Site getSite() {
        return site;
    }

    public static void main(String[] args) {
        Spider.create(new GithubRepoPageProcessor()).addUrl("https://github.com/code4craft").thread(5).run();
    }

    @Test
    public void insertTest() {
        //爬取小说基本信息
        String url = "http://jinyong.zishiba.com/ludingji/";
        Document doc = cn.book.bus.utils.JsoupUtil.getDoc(url);
        String fictionName = Optional.of(cn.book.bus.utils.JsoupUtil.getTexts(doc, "h1")).filter(list -> !list.isEmpty())
                .map(list -> list.get(0))
                .orElse("Default Title");
        String brief = cn.book.bus.utils.JsoupUtil.getContent(doc, "m-bkcont");
        String author = "";
        String type = "武侠";
        String newest = "";
        // 提取最近更新
        Element updateElement = doc.select("p:contains(最近更新：)").first();
        if (updateElement != null) {
            newest = updateElement.text().replace("最近更新：", "").trim();
        }

        // 提取作者
        Element authorElement = doc.select("p:contains(作者：)").first();
        if (authorElement != null) {
            author = authorElement.text().replace("作者：", "").trim();
        }

        String state = "";
        // 提取状态
        Element statusElement = doc.select("p:contains(状态：)").first();
        if (statusElement != null) {
            state = statusElement.text().replace("状态：", "").trim();
        }
        String img = JsoupUtil.getAttributes(doc, ".book-img img", "src").stream().findFirst().get();

    }
}